#include <boost/algorithm/string.hpp>
#include <strings.h>
#include <sstream>
#include <iostream>
#include <fstream>
#include <vector>
#include <map>

using namespace std;
using namespace boost;



map<string, long> lookUpDict;
map<string, long>::iterator lookUpDictIter;

//example
//clueweb09-en0000-40-00000

//the spam rank file
const char* spamRankDataFileName = "/data1/team/weijiang/machine-learning-project-related/auxFiles/clueweb09spam.Fusion";

//the index for spam rank file
const char* indexForSpamRankDictFileName = "/data5/team/weijiang/the_new_trip_of_feature_generation/dictForSpamRank";

//the backup index file for spam rank file
string backupIndexFilePathForSpamRankDict = "/data1/team/weijiang/machine-learning-project-related/auxFiles/backup-2012-10-03-";


int backupDict(string backupDictName){
	  cout << "backup dict" << endl;
	  ofstream backupOutputfile(backupDictName.c_str());

	  //traversal of the whole map
	  for(lookUpDictIter = lookUpDict.begin(); lookUpDictIter != lookUpDict.end(); lookUpDictIter++)
	  {
		  //cout << lookUpDictIter->first << " " << lookUpDictIter->second << endl;
		  backupOutputfile << lookUpDictIter->first << " " << lookUpDictIter->second << endl;
	  }
	  cout << lookUpDict.size() << " tuples have been dumped(for backup)." << endl;
	  backupOutputfile.close();
	  return 0;
}

// obtaining file size
int obtainFileSize() {
  long begin,end;

  ifstream myfile(spamRankDataFileName);

  begin = myfile.tellg();

  myfile.seekg (0, ios::end);

  end = myfile.tellg();

  myfile.close();

  cout << "size is: " << (end-begin) << " bytes.\n";

  return 0;
}

int buildHash_Map(){

	map<string, long> lookUpDict;
	map<string, long>::iterator loolUpDictIter;

	lookUpDict["00"] = 0;
	lookUpDict["01"] = 2;
	//cout << mymap.size();


	loolUpDictIter = lookUpDict.find("01");
	cout << loolUpDictIter->second << endl;

	/*
	if(mymap.find("00") != mymap.end()){
		cout << "true" << endl;
	}
	*/

	return 0;
}


// build the parsed tree
int urlParser(){
	  const int BACKUP_THRESHOLD = 50000000;
	  //const int BACKUP_THRESHOLD = 100000;

	  cout << "build dict" << endl;
	  //long offset = 0;

	  // This is for backup use.
	  long lineCounter = 0;


	  string line;
	  string trimedLine;
	  vector<string> elements;
	  vector<string> elements2;
	  ifstream myinputFile(spamRankDataFileName);

	  //This is the beginning value of the language tag.
	  string currentLanguageTag = "en0000";
	  int currentLanguageTagCounter = 1;

	  //This is the beginning value of the volumn tag.
	  string currentVolumn = "00";
	  int currentVolumnCounter = 0;

	  string currentWebPageNumber = "00000";


	  // initial: store 2 numbers for sequential scan.
	  lookUpDict[currentLanguageTag] = 0;
	  lookUpDict[currentLanguageTag + "-" + currentVolumn] = 0;

	  if (myinputFile.is_open())
	  {
	    while ( myinputFile.good() )
	    {
	      //Maybe this value will be stored.
	      const long offset = myinputFile.tellg();
	      if(offset < 0){
	    	  throw "-1";
	      }
	      getline (myinputFile,line);



	      if (lineCounter % BACKUP_THRESHOLD == 0){
	    	  cout << "lineCounter:" << lineCounter << endl;
	    	  stringstream temp;
	    	  temp << lineCounter;
	    	  backupIndexFilePathForSpamRankDict += temp.str() + "-dictForSpamRank";
	    	  backupDict( backupIndexFilePathForSpamRankDict );
	      }

	      lineCounter ++;

	      if (line != ""){

		      trim(line);
		      split(elements, line, is_any_of(" ") );
		      split(elements2, elements[1], is_any_of("-") );
		      trim(elements2[0]);
		      trim(elements2[1]);
		      trim(elements2[2]);
		      trim(elements2[3]);

		      //cout << "elements2[1]:" << elements2[1] << " " << "currentLanguageTag:" << currentLanguageTag << endl;
		      //cout << "elements2[2]:" << elements2[2] << " " << "currentVolumn:" << currentVolumn << endl;
		      //cout << "elements2[3]:" << elements2[3] << " " << "currentWebPageNumber:" << currentWebPageNumber << endl;


		      if (elements2[1] == currentLanguageTag)
		      {
			      if(elements2[2] == currentVolumn)
			      {

			    	  currentVolumnCounter++;

			      }
			      else
			      {
			    	  cout << currentLanguageTag << "-" << currentVolumn << ":" << currentVolumnCounter << endl;

			    	  currentVolumn = elements2[2];
			    	  currentVolumnCounter = 1;
			    	  currentLanguageTagCounter++;


			    	  lookUpDict[currentLanguageTag + "-" + currentVolumn] = offset;
			      }
		      }
		      else
		      {
		    	  //dump the last volumn.
		    	  cout << currentLanguageTag << "-" << currentVolumn << ":" << currentVolumnCounter << endl;
		    	  currentVolumn = elements2[2];
		    	  currentVolumnCounter = 1;



		    	  //dump the previous language tag.
		    	  cout << currentLanguageTag << ":" << currentLanguageTagCounter << endl;

		    	  currentLanguageTag = elements2[1];
		    	  currentLanguageTagCounter = 1;

		    	  lookUpDict[currentLanguageTag] = offset;
		    	  lookUpDict[currentLanguageTag + "-" + currentVolumn] = offset;
		      }
		      //cout << elements2[1] << ":" << elements2[2] << ":" << elements2[3] << endl;
	      }
	      else{
	    	  //final wrap up

	    	  //dump the last volumn.
	    	  cout << currentLanguageTag << "-" << currentVolumn << ":" << currentVolumnCounter << endl;




	    	  //dump the previous language tag.
	    	  cout << currentLanguageTag << ":" << currentLanguageTagCounter << endl;



	      }
	    }
	    myinputFile.close();
	  }
	  else{
		  cout << "Unable to open file" << endl;
	  }
	  /*
	  //traversal of the whole map
	  cout << "*****" << endl;
	  for(lookUpDictIter = lookUpDict.begin(); lookUpDictIter != lookUpDict.end(); lookUpDictIter++)
	  {
		  cout << lookUpDictIter->first << " " << lookUpDictIter->second << endl;
	  }
	  */

	  cout << "# of Entries:" << lookUpDict.size() << endl;
	  cout << "Build Tree in Memory, Done!" << endl;
	  return 0;
}

int getHamProbablity(string lookUpWARCTRECID){
	  //Let's do some real work.
	  ifstream myinputFileActualLookUp(spamRankDataFileName);
	  string line = "";



	  vector<string> elementsForLookUpString;
	  split(elementsForLookUpString, lookUpWARCTRECID, is_any_of("-") );
	  string searchString = elementsForLookUpString[1] + "-" + elementsForLookUpString[2];
	  //cout << "search string:" << searchString << endl;

	  if(lookUpDict.find(searchString) != lookUpDict.end()){


		  lookUpDictIter = lookUpDict.find(searchString);
		  //cout << lookUpDictIter->second << endl;

		  cout << "Find Entry:" << lookUpDictIter->second << endl;

		  //cout << "In seekg" << endl;
		  //currently no ending bound.
		  myinputFileActualLookUp.seekg(lookUpDictIter->second, ios::beg);
		  //cout << "Out seekg" << endl;

		  //clueweb09-en0045-40-00000
		  while ( myinputFileActualLookUp.good() )
		  {
			  //cout << "file is good" << endl;
			  //Let's do a sequential search here.
			  getline (myinputFileActualLookUp,line);

			  if(line != ""){
				  trim(line);
				  //cout << "line:" << line << endl;


				  vector<string> elementsForGoal1;
				  vector<string> elementsForGoal2;

				  split(elementsForGoal1, line, is_any_of(" ") );
				  split(elementsForGoal2, elementsForGoal1[1], is_any_of("-") );

				  string currentMatchingString = elementsForGoal2[0] + "-" + elementsForGoal2[1] + "-" + elementsForGoal2[2] + "-" + elementsForGoal2[3];

				  //cout << "currentMatchingString:" << currentMatchingString << endl;
				  //cout << "lookUpString:" << lookUpString << endl;
				  if (currentMatchingString == lookUpWARCTRECID){
					  cout << "WARC TREC ID:" << lookUpWARCTRECID << endl;
					  cout << "ham probability:" << elementsForGoal1[0] << "%" << endl;
					  break;
				  }
			  }
		  }
	  }
}




int dumpDict(){
	  cout << "dump dict" << endl;
	  ofstream outputfile(indexForSpamRankDictFileName);
	  //traversal of the whole map
	  for(lookUpDictIter = lookUpDict.begin(); lookUpDictIter != lookUpDict.end(); lookUpDictIter++)
	  {
		  //cout << lookUpDictIter->first << " " << lookUpDictIter->second << endl;
		  outputfile << lookUpDictIter->first << " " << lookUpDictIter->second << endl;
	  }
	  cout << lookUpDict.size() << " tuples have been dumped." << endl;
	  outputfile.close();
	  return 0;
}

int loadDict(){
	  string line = "";
	  ifstream inputfile(indexForSpamRankDictFileName);
	  vector<string> elements;

	  while ( inputfile.good() )
	  {
		  getline (inputfile,line);
		  if(line != ""){
		      trim(line);
		      split(elements, line, is_any_of(" ") );
		      // If this is the case, fu*k
		      lookUpDict[ elements[0] ] = atol( elements[1].c_str() );
		  }
	  }
	  inputfile.close();


	  //traversal of the whole map
	  for(lookUpDictIter = lookUpDict.begin(); lookUpDictIter != lookUpDict.end(); lookUpDictIter++)
	  {
		  cout << lookUpDictIter->first << " " << lookUpDictIter->second << endl;
	  }

	  if(lookUpDict.size() == 0){
		  cout << "Load Task is NOT OK" << endl;
		  exit(1);
	  }
	  else{
		  cout << lookUpDict.size() << " tuples have been loaded." << endl;
	  }
	  return 0;
}

int loadPopularSiteReport(){
      const char* dataFileName = "/data1/team/weijiang/dataset/top-1m.csv";
      map<string, string> lookUpDictForPopularSite;
      map<string, string>::iterator lookUpDictIterForPopularSite;
	  string currentLine = "";
	  ifstream inputfileForPopularSites(dataFileName);
	  vector<string> elementsForPopularSites;

	  while ( inputfileForPopularSites.good() )
	  {
		  getline (inputfileForPopularSites,currentLine);
		  if(currentLine != ""){
		      trim(currentLine);
		      split(elementsForPopularSites, currentLine, is_any_of(",") );
		      // If this is the case, fu*k
		      lookUpDictForPopularSite[ elementsForPopularSites[1] ] = elementsForPopularSites[0];
		  }
	  }
	  inputfileForPopularSites.close();

	  //traversal of the whole map
	  for(lookUpDictIterForPopularSite = lookUpDictForPopularSite.begin(); lookUpDictIterForPopularSite != lookUpDictForPopularSite.end(); lookUpDictIterForPopularSite++)
	  {
		  cout << lookUpDictIterForPopularSite->first << " " << lookUpDictIterForPopularSite->second << endl;
	  }
	  cout << lookUpDictForPopularSite.size() << " tupuls have been loaded." << endl;

	  return 0;
}

int assignBM25ScoreToTermFeatureFile(map<string, double> &lookUpDictForBM25Score){
	  //Load the term_features file as well and append the score to each of the document.

      const char* oldFeatureFileName = "/data1/team/weijiang/machine-learning-project-related/featuresGenerated/term_features.txt";
      const char* newFeatureFileName = "/data1/team/weijiang/machine-learning-project-related/featuresGenerated/term_features_new.txt";

      string oldLine = "";
      string newLine = "";
      string mapKey = "";
      double mapValue = 0.0;

      int totalCounter = 0;
      int notZeroCounter = 0;
      int zeroCounter = 0;

	  ifstream inputfileForFeatures(oldFeatureFileName);
	  ofstream outputfileForFeatures(newFeatureFileName);

	  vector<string> elementsOfFeatureLine;

	  while ( inputfileForFeatures.good() )
	  {
		  getline (inputfileForFeatures,oldLine);
		  if(oldLine != ""){
		      trim(oldLine);
		      split(elementsOfFeatureLine, oldLine, is_any_of(" ") );
		      mapKey = elementsOfFeatureLine[0] + "_" + elementsOfFeatureLine[1];
		      mapValue = lookUpDictForBM25Score[mapKey];
		      if(mapValue != 0){
		    	  notZeroCounter++;
		      }
		      else{
		    	  zeroCounter++;
		    	  cout << "mapKey:" << mapKey << " " << "mapValue:" <<mapValue << endl;
		      }
		      //cout << "mapKey:" << mapKey << endl;
		      //cout << "mapValue:" << mapValue << endl;
		      ostringstream mapValueStringStream;
		      mapValueStringStream << mapValue;
		      newLine = elementsOfFeatureLine[0] + " " + elementsOfFeatureLine[1] + " " + elementsOfFeatureLine[2] + " " + elementsOfFeatureLine[3] + " " + mapValueStringStream.str();
		      //cout << newLine << endl;
		      outputfileForFeatures << newLine << endl;
		      totalCounter++;
		  }
	  }
	  inputfileForFeatures.close();
	  outputfileForFeatures.close();

	  cout << "this is for testing:" << endl;
	  cout << "lookUpDictForBM25Score['1024px_0']:" << lookUpDictForBM25Score["1024px_0"] << endl;
	  cout << "totalCounter:" << totalCounter << endl;
	  cout << "notZeroCounter:" << notZeroCounter << endl;
	  cout << "zeroCounter:" << zeroCounter << endl;
	  cout << "Job Done." << endl;

	  return 0;
}

int loadBM25ScoreFromDisk(){
	  //Load all the BM25 scores into the dict first.

      const char* BM25FeatureFileName = "/data1/team/weijiang/machine-learning-project-related/featuresGenerated/term_features_BM25.txt";
      map<string, double> lookUpDictForBM25Score;
      map<string, double>::iterator lookUpDictIterForBM25Score;
      string mapKey = "";
      double mapValue = 0.0;

      string currentLine = "";
	  ifstream inputfileForBM25Features(BM25FeatureFileName);
	  vector<string> elementsForBM25FeatureLine;

	  while ( inputfileForBM25Features.good() )
	  {
		  getline (inputfileForBM25Features,currentLine);
		  if(currentLine != ""){
		      trim(currentLine);
		      split(elementsForBM25FeatureLine, currentLine, is_any_of(" ") );
		      mapKey = elementsForBM25FeatureLine[0] + "_" + elementsForBM25FeatureLine[1];
		      mapValue = atof(elementsForBM25FeatureLine[2].c_str());
		      lookUpDictForBM25Score[mapKey] = mapValue;
		  }
	  }
	  inputfileForBM25Features.close();


	  cout << lookUpDictForBM25Score.size() << " tupuls have been loaded." << endl;

	  assignBM25ScoreToTermFeatureFile(lookUpDictForBM25Score);



	  return 0;
}



int main(int argc, char* argv[]){
	//wei comment: if this time is not good, then I just check it carefully again.
	if (argc != 2){
		cout << "argc = " << argc << endl;
		cout << "usage:" << "have an input flag: 1 for build tree, 2 for load tree and search" << endl;
	}
	else{
		if(*argv[1] == '1'){
			//It takes 2.5 hours to build up the tree.
			cout << "Build Task" << endl;
			urlParser();
			dumpDict();
		}
		else if(*argv[1] == '2'){

			cout << "Load Task" << endl;
			string line = "";
			loadDict();

			cout << "Search Task" << endl;
			while(true){
				cout << "Enter Valid WARC TREC ID:";
				getline(cin, line);
				trim(line);
				getHamProbablity(line);
			}
		}
		else if(*argv[1] == '3'){
			loadPopularSiteReport();
		}
		else if(*argv[1] == '4'){
			loadBM25ScoreFromDisk();
		}
		else{
			cout << "Exit" << endl;
			exit(0);
		}
	}


	/*
	// It takes 2.5 hours to build up the tree.
	// Wei: if only takes 2.5 hours, that is better than I thought.
	cout << "Build Task" << endl;
	buildDict();
	dumpDict();
	*/





	/*
	cout << "Load Task" << endl;
	string line = "";
	loadDict();

	while(true){
		cout << "Enter Valid WARC TREC ID:";
		getline(cin, line);
		trim(line);
		getHamProbablity(line);
	}
	*/

	return 0;
}
